const cluster = require('cluster');
const numCPUs = require('os').cpus().length;
var got = require('got');
var assert = require('assert')
const cheerio = require('cheerio')
const {sleep} = require('../util.js')
var amqp = require('amqplib');
const puppeteer = require('puppeteer');
// const tough = require('tough-cookie');
const {AMQP_ADDRESS,QUEUE_SINGLE_WK_NAME,QUEUE_SINGLE_RET_NAME} = require('../config.js')

// 启动多少的进程
var options = process.argv;
const numWorks = Number(options[2]) || numCPUs - 1
const customer_count = 10
const got_count_max = 100
global.current_got_count = 0

const goto_page = async (page,keyword) => {
  let search_key = `"${keyword}"`

      // let key_word = encodeURIComponent('"'+name+'"')
  await page.focus("#query, #upquery")
  await page.waitFor(100);

  await page.keyboard.down('ControlLeft');
  await page.keyboard.press('KeyA');
  await page.keyboard.press('Delete');
  await page.keyboard.up('ControlLeft');

  await page.keyboard.type(search_key)
  await page.keyboard.press("Enter")
  // await page.waitForResponse(response => response.url().match(/https:\/\/www.sogou.com\/web\?query=/) && response.status() === 200);
  // await page.waitForSelector("#query, #upquery")
  await page.waitForNavigation({waitUntil:'domcontentloaded'})

  // let html = await page.evaluate('document.body.textContent');
  let html = await page.content()
  let $ = cheerio.load(html)
  assert.equal($("#query, #upquery").attr('value'),search_key)
  if (html.match(/抱歉，没有找到与/)){
    return [0,0,0]
  }

  let ems = $('.results em').text()
  let keyword_nums = 0
  if (ems.match(new RegExp(keyword,'g')))
     keyword_nums = ems.match(new RegExp(keyword,'g')).length

  let nums = $('.num-tips').text()
  nums = nums.replace(/,/g,"").match(/[0-9]\d*/)[0]
  nums = Number(nums)
  return [keyword_nums,nums]
}

const got_page = async (headers,keyword) => {
  let search_key = `"${keyword}"`

  let query_key = encodeURIComponent(search_key)
  let response = await got('https://www.sogou.com/web?ie=utf8&query='+ query_key,{headers:headers,timeout:10000})

  // let html = await page.evaluate('document.body.textContent');
  let html = response.body
  let $ = cheerio.load(html)
  assert.equal($("#query, #upquery").attr('value'),search_key)
  if (html.match(/抱歉，没有找到与/)){
    return [0,0,0]
  }

  let ems = $('.results em').text()
  let keyword_nums = 0
  if (ems.match(new RegExp(keyword,'g')))
     keyword_nums = ems.match(new RegExp(keyword,'g')).length

  let nums = $('.num-tips').text()
  nums = nums.replace(/,/g,"").match(/[0-9]\d*/)[0]
  nums = Number(nums)
  return [keyword_nums,nums]
}

const got_customer = async (headers,got_id)=>{
    try {
      while (true) {
  	    try{
          let msg = await ch.get(q_wk.queue,{noAck:false})
          if (!msg){
            await sleep(1000)
            continue
          }
          let data = JSON.parse(msg.content.toString())
          let name = data['name']
          let keyword = data['keyword']
          let search_key = `"${keyword}"`
          if (search_key.length>40){
            console.log('lenght > 40',search_key)
            await ch.ack(msg)
            continue
          }
          while (true){
            try {
              var [keyword_nums,nums] = await got_page(headers,keyword)
              break
            } catch (err) {
              console.error(got_id,'got',err.message)
              try {
                ch.nack(msg)
              } catch (err) {} finally {
                global.current_got_count -= 1
                return
              }
            }
          }
          await ch.sendToQueue(q_ret.queue,
                  Buffer.from(JSON.stringify({
                    keyword:keyword,
                    keyword_match:keyword_nums,
                    sogou_match:nums,
                  })));

          await ch.ack(msg)
          let flt = (new Date() - start_time)/succ_count/1000
          succ_count+=1
          console.log(new Date,process.pid,'suc:',succ_count,'gID:',got_id,'gC:',global.current_got_count,'eT:',flt.toFixed(2),'match:',keyword_nums)
          } catch (err) {
        	  console.log(err.message)
        	  sleep(2000)
        	}
      }

    } catch (err) {
      console.trace(err.message);
      global.current_got_count -= 1
      return
    }
}

const customer = async ()=>{
  var got_id = 0
    try {
      var page = await init_page()
      // await page.setCacheEnabled(false)
      await page.goto('https://www.sogou.com/',{waitUntil: 'networkidle2',timeout: 240000})
      while (true) {
  	    try{
          let msg = await ch.get(q_wk.queue,{noAck:false})
          if (!msg){
            await sleep(1000)
            continue
          }
          let data = JSON.parse(msg.content.toString())
          let keyword = data['keyword']
          let search_key = `"${keyword}"`
          if (search_key.length>40){
            console.log('lenght > 40',search_key)
            await ch.ack(msg)
            continue
          }
          while (true){
            try {
              var [keyword_nums,nums] = await goto_page(page,keyword)
              break
            } catch (err) {
              console.trace(err.message)
              try {
                let cookies = await page.cookies()
                await page.deleteCookie(...cookies)
                await page.close()
              } catch (err) {}
              try {
                page = await init_page()
                await page.goto('https://www.sogou.com/',{waitUntil: 'networkidle2',timeout: 240000})
              } catch (err) {
                try {
                  await init_browser()
                  page = await init_page()
                  await page.goto('https://www.sogou.com/',{waitUntil: 'networkidle2',timeout: 240000})
                } catch (err) {
                  console.trace(err.message)
                }
              }
              fail_count += 1
              console.log('sucee:',succ_count,'faile:',fail_count)
              }

          }
          await ch.sendToQueue(q_ret.queue,
                  Buffer.from(JSON.stringify({
                    keyword:keyword,
                    keyword_match:keyword_nums,
                    sogou_match:nums,
                  })));

          await ch.ack(msg)
          let flt = (new Date() - start_time)/succ_count/1000
          succ_count+=1
          console.log(new Date,process.pid,'succ:',succ_count,'faile:',fail_count,'eachTime',flt.toFixed(2),'match:',keyword_nums)

          leader_flag = true
          // const cookieJar = new tough.CookieJar();
          let cookies = await page.cookies()
          let headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"
          }
          let myCookie = []
          cookies.forEach(elem => {
            myCookie.push(elem['name']+'='+elem['value'])
          });
          headers['cookie'] = myCookie.join(';')
          for(i=0;i<customer_count;i++){
            got_customer(headers,got_id)
            global.current_got_count += 1
            got_id += 1
          }


            // 重开获取cookie
            try {
              let cookies = await page.cookies()
              await page.deleteCookie(...cookies)
              await page.close()
            } catch (err) {}
            try {
              page = await init_page()
              await page.goto('https://www.sogou.com/',{waitUntil: 'networkidle2',timeout: 240000})
            } catch (err) {console.trace(err.message)}
            await sleep(10000)
          } catch (err) {
            if (err.message.match(/Channel closed/)){
              await sleep(10000)
              await connect()
            }
        	  console.log(err.message)
        	  sleep(2000)
        	}
      }

    } catch (err) {
      console.trace(err.message);
      return
    }
}



const test = async (keyword)=>{
      await init_browser()
      var page = await init_page()
      await page.setCacheEnabled(false)
      await page.goto('https://www.sogou.com/',{waitUntil: 'networkidle2',timeout: 240000})
      let search_key = `"${keyword}"`
      while (true){
        try {
          var [keyword_nums,nums] = await goto_page(page,keyword)
          console.log(keyword_nums,nums)
          await page.waitFor(1000000)
        } catch (err) {
          console.trace('line:' + err.lineNumber,'message:' + err.message)
          await sleep(100)
          try {
            let cookies = await page.cookies()
            await page.deleteCookie(...cookies)
            await page.close()
          } catch (err) {}
          try {
            page = await init_page()
            await page.goto('https://www.sogou.com/',{waitUntil: 'networkidle2',timeout: 240000})
          } catch (err) {
            console.trace(err)
          }

        }
      }

}

// 定义全局变量，重连是统一用
var conn,ch,q_wk,q_ret
const connect = async ()=>{
  try {
      conn = await amqp.connect(AMQP_ADDRESS)
      ch = await conn.createChannel()
      q_wk = await ch.assertQueue(QUEUE_SINGLE_WK_NAME, {durable: false});
      q_ret = await ch.assertQueue(QUEUE_SINGLE_RET_NAME, {durable: false});
  } catch (err) {
    console.trace(err.message);
  }
}

var browserWSEndpoint
const init_browser = async () => {
  var browser = await puppeteer.launch({
    handleSIGINT: true, // so Chrome doesn't exit when we quit Node.
    headless: true, // to see what's happening
    ignoreHTTPSErrors : true,
    args: ['--ignore-certificate-errors']
  });
  browserWSEndpoint = browser.wsEndpoint()
  return browser
}
const init_page = async () => {
  var browser = await puppeteer.connect({browserWSEndpoint});
  const page = await browser.newPage();
  await page.setUserAgent("Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36")
  // 允许拦截请求
  await page.setRequestInterception(true);
  page.on('request', request => {
    // console.log(request.url());
    // 拦截
    // if (['image', 'stylesheet', 'font', 'script'].indexOf(request.resourceType()) !== -1)
    if ([ 'image','font'].indexOf(request.resourceType()) !== -1)
        request.abort();
    else
        request.continue();
  });
  return page
}

// 启动100个采集器
var succ_count = 0
var fail_count = 0
  var start_time = new Date()
const main = async ()=>{
  await connect(); 
  conn.on('close', async ()=> {
    console.error("[AMQP] close");
  })
  conn.on('error', async ()=> {
    console.error("[AMQP] error");
    process.exit()
  })

  while(true){
    var browser = await init_browser();

    customer()
    // got_customer()
      // 按次数重启
    while (true){
      await sleep(10000)
      if (new Date() - start_time > 60 * 1000 * 60 * 1){
        process.exit()
        // restart
      }
    }
  }
}

// 启动多个子线程
if (cluster.isMaster) {
  for (let i = 0; i < numWorks-1; i++) {
    cluster.fork();
  }
}


main()

// test('成都易达出行科技有限公司','阻燃剂')
